
import os
from IPython.display import display, HTML
display(HTML("<style>.container { width:100% !important; }</style>"))


# If you want to run this notebook locally, copy the output of the next cell into the next line of this cell.
# antenna = "004"
# csv_folder = '/lustre/aoc/projects/hera/H5C/H5C_Notebooks/_rtp_summary_'
# auto_metrics_folder = '/lustre/aoc/projects/hera/H5C/H5C_Notebooks/auto_metrics_inspect'
# os.environ["ANTENNA"] = antenna
# os.environ["CSV_FOLDER"] = csv_folder
# os.environ["AUTO_METRICS_FOLDER"] = auto_metrics_folder


# Use environment variables to figure out path to the csvs and auto_metrics
antenna = str(int(os.environ["ANTENNA"]))
csv_folder = os.environ["CSV_FOLDER"]
auto_metrics_folder = os.environ["AUTO_METRICS_FOLDER"]
print(f'antenna = "{antenna}"')
print(f'csv_folder = "{csv_folder}"')
print(f'auto_metrics_folder = "{auto_metrics_folder}"')

antenna = "239"
csv_folder = "/home/obs/src/H6C_Notebooks/_rtp_summary_"
auto_metrics_folder = "/home/obs/src/H6C_Notebooks/auto_metrics_inspect"


display(HTML(f'<h1 style=font-size:50px><u>Antenna {antenna} Report</u><p></p></h1>'))


import numpy as np
import pandas as pd
pd.set_option('display.max_rows', 1000)
import glob
import re
from hera_notebook_templates.utils import status_colors, Antenna


# load csvs and auto_metrics htmls in reverse chronological order
csvs = sorted(glob.glob(os.path.join(csv_folder, 'rtp_summary_table*.csv')))[::-1]
print(f'Found {len(csvs)} csvs in {csv_folder}')
auto_metric_htmls = sorted(glob.glob(auto_metrics_folder + '/auto_metrics_inspect_*.html'))[::-1]
print(f'Found {len(auto_metric_htmls)} auto_metrics notebooks in {auto_metrics_folder}')

Found 25 csvs in /home/obs/src/H6C_Notebooks/_rtp_summary_
Found 25 auto_metrics notebooks in /home/obs/src/H6C_Notebooks/auto_metrics_inspect


# Per-season options
mean_round_modz_cut = 4
dead_cut = 0.4
crossed_cut = 0.0

def jd_to_summary_url(jd):
    return f'https://htmlpreview.github.io/?https://github.com/HERA-Team/H6C_Notebooks/blob/main/_rtp_summary_/rtp_summary_{jd}.html'

def jd_to_auto_metrics_url(jd):
    return f'https://htmlpreview.github.io/?https://github.com/HERA-Team/H6C_Notebooks/blob/main/auto_metrics_inspect/auto_metrics_inspect_{jd}.html'


this_antenna = None
jds = []

# parse information about antennas and nodes
for csv in csvs:
    df = pd.read_csv(csv)
    for n in range(len(df)):
        # Add this day to the antenna
        row = df.loc[n]
        if isinstance(row['Ant'], str) and '<a href' in row['Ant']:
            antnum = int(row['Ant'].split('</a>')[0].split('>')[-1]) # it's a link, extract antnum
        else:
            antnum = int(row['Ant'])
        if antnum != int(antenna):
            continue
        
        if np.issubdtype(type(row['Node']), np.integer):
            row['Node'] = str(row['Node'])
        if type(row['Node']) == str and row['Node'].isnumeric():
            row['Node'] = 'N' + ('0' if len(row['Node']) == 1 else '') + row['Node']
            
        if this_antenna is None:
            this_antenna = Antenna(row['Ant'], row['Node'])
        jd = [int(s) for s in re.split('_|\.', csv) if s.isdigit()][-1]
        jds.append(jd)
        this_antenna.add_day(jd, row)
        break


# build dataframe
to_show = {'JDs': [f'<a href="{jd_to_summary_url(jd)}" target="_blank">{jd}</a>' for jd in jds]}
to_show['A Priori Status'] = [this_antenna.statuses[jd] for jd in jds]

df = pd.DataFrame(to_show)

# create bar chart columns for flagging percentages:
bar_cols = {}
bar_cols['Auto Metrics Flags'] = [this_antenna.auto_flags[jd] for jd in jds]
bar_cols[f'Dead Fraction in Ant Metrics (Jee)'] = [this_antenna.dead_flags_Jee[jd] for jd in jds]
bar_cols[f'Dead Fraction in Ant Metrics (Jnn)'] = [this_antenna.dead_flags_Jnn[jd] for jd in jds]
bar_cols['Crossed Fraction in Ant Metrics'] = [this_antenna.crossed_flags[jd] for jd in jds]
bar_cols['Flag Fraction Before Redcal'] = [this_antenna.flags_before_redcal[jd] for jd in jds]
bar_cols['Flagged By Redcal chi^2 Fraction'] = [this_antenna.redcal_flags[jd] for jd in jds]
for col in bar_cols:
    df[col] = bar_cols[col]

z_score_cols = {}
z_score_cols['ee Shape Modified Z-Score'] = [this_antenna.ee_shape_zs[jd] for jd in jds]
z_score_cols['nn Shape Modified Z-Score'] = [this_antenna.nn_shape_zs[jd] for jd in jds]
z_score_cols['ee Power Modified Z-Score'] = [this_antenna.ee_power_zs[jd] for jd in jds]
z_score_cols['nn Power Modified Z-Score'] = [this_antenna.nn_power_zs[jd] for jd in jds]
z_score_cols['ee Temporal Variability Modified Z-Score'] = [this_antenna.ee_temp_var_zs[jd] for jd in jds]
z_score_cols['nn Temporal Variability Modified Z-Score'] = [this_antenna.nn_temp_var_zs[jd] for jd in jds]
z_score_cols['ee Temporal Discontinuties Modified Z-Score'] = [this_antenna.ee_temp_discon_zs[jd] for jd in jds]
z_score_cols['nn Temporal Discontinuties Modified Z-Score'] = [this_antenna.nn_temp_discon_zs[jd] for jd in jds]
for col in z_score_cols:
    df[col] = z_score_cols[col]

ant_metrics_cols = {}
ant_metrics_cols['Average Dead Ant Metric (Jee)'] = [this_antenna.Jee_dead_metrics[jd] for jd in jds]
ant_metrics_cols['Average Dead Ant Metric (Jnn)'] = [this_antenna.Jnn_dead_metrics[jd] for jd in jds]
ant_metrics_cols['Average Crossed Ant Metric'] = [this_antenna.crossed_metrics[jd] for jd in jds]
for col in ant_metrics_cols:
    df[col] = ant_metrics_cols[col]

redcal_cols = {}
redcal_cols['Median chi^2 Per Antenna (Jee)'] = [this_antenna.Jee_chisqs[jd] for jd in jds]
redcal_cols['Median chi^2 Per Antenna (Jnn)'] = [this_antenna.Jnn_chisqs[jd] for jd in jds]   
for col in redcal_cols:
    df[col] = redcal_cols[col]

# style dataframe
table = df.style.hide_index()\
          .applymap(lambda val: f'background-color: {status_colors[val]}' if val in status_colors else '', subset=['A Priori Status']) \
          .background_gradient(cmap='viridis', vmax=mean_round_modz_cut * 3, vmin=0, axis=None, subset=list(z_score_cols.keys())) \
          .background_gradient(cmap='bwr_r', vmin=dead_cut-.25, vmax=dead_cut+.25, axis=0, subset=list([col for col in ant_metrics_cols if 'dead' in col.lower()])) \
          .background_gradient(cmap='bwr_r', vmin=crossed_cut-.25, vmax=crossed_cut+.25, axis=0, subset=list([col for col in ant_metrics_cols if 'crossed' in col.lower()])) \
          .background_gradient(cmap='plasma', vmax=4, vmin=1, axis=None, subset=list(redcal_cols.keys())) \
          .applymap(lambda val: 'font-weight: bold' if val < dead_cut else '', subset=list([col for col in ant_metrics_cols if 'dead' in col.lower()])) \
          .applymap(lambda val: 'font-weight: bold' if val < crossed_cut else '', subset=list([col for col in ant_metrics_cols if 'crossed' in col.lower()])) \
          .applymap(lambda val: 'font-weight: bold' if val > mean_round_modz_cut else '', subset=list(z_score_cols.keys())) \
          .applymap(lambda val: 'color: red' if val > mean_round_modz_cut else '', subset=list(z_score_cols.keys())) \
          .bar(subset=list(bar_cols.keys()), vmin=0, vmax=1) \
          .format({col: '{:,.4f}'.format for col in z_score_cols}) \
          .format({col: '{:,.4f}'.format for col in ant_metrics_cols}) \
          .format('{:,.2%}', na_rep='-', subset=list(bar_cols.keys())) \
          .set_table_styles([dict(selector="th",props=[('max-width', f'70pt')])])


display(HTML(f'<h2>Antenna {antenna}, Node {this_antenna.node}:</h2>'))
HTML(table.render(render_links=True, escape=False))


htmls_to_display = []
for am_html in auto_metric_htmls:
    html_to_display = ''
    # read html into a list of lines
    with open(am_html) as f:
        lines = f.readlines()
    
    # find section with this antenna's metric plots and add to html_to_display
    jd = [int(s) for s in re.split('_|\.', am_html) if s.isdigit()][-1]
    try:
        section_start_line = lines.index(f'<h2>Antenna {antenna}: {jd}</h2>\n')
    except ValueError:
        continue
    html_to_display += lines[section_start_line].replace(str(jd), f'<a href="{jd_to_auto_metrics_url(jd)}" target="_blank">{jd}</a>')
    for line in lines[section_start_line + 1:]:
        html_to_display += line
        if '<hr' in line:
            htmls_to_display.append(html_to_display)
            break


for i, html_to_display in enumerate(htmls_to_display):
    if i == 100:
        break
    display(HTML(html_to_display))

JDs	A Priori Status	Auto Metrics Flags	Dead Fraction in Ant Metrics (Jee)	Dead Fraction in Ant Metrics (Jnn)	Crossed Fraction in Ant Metrics	Flag Fraction Before Redcal	Flagged By Redcal chi^2 Fraction	ee Shape Modified Z-Score	nn Shape Modified Z-Score	ee Power Modified Z-Score	nn Power Modified Z-Score	ee Temporal Variability Modified Z-Score	nn Temporal Variability Modified Z-Score	ee Temporal Discontinuties Modified Z-Score	nn Temporal Discontinuties Modified Z-Score	Average Dead Ant Metric (Jee)	Average Dead Ant Metric (Jnn)	Average Crossed Ant Metric	Median chi^2 Per Antenna (Jee)	Median chi^2 Per Antenna (Jnn)
2460008	RF_ok	0.00%	0.00%	0.00%	0.00%	-	-	-1.285654	-0.925517	-0.222293	0.137222	-0.145172	-0.059215	-0.680210	-0.165738	0.6360	0.6461	0.3267	nan	nan
2460007	RF_ok	0.00%	0.00%	0.00%	0.00%	-	-	-0.841091	-0.673932	0.238629	0.270974	-0.287368	-0.414706	-0.680511	0.477846	0.6063	0.6177	0.3446	nan	nan
2459999	RF_ok	0.00%	0.00%	0.00%	0.00%	-	-	nan	nan	nan	nan	nan	nan	nan	nan	0.5802	0.6033	0.3243	nan	nan
2459998	RF_ok	0.00%	0.00%	0.00%	0.00%	-	-	-0.824571	-0.808305	-0.256907	-0.029079	-0.518097	-0.741572	-0.144957	0.327153	0.5885	0.6043	0.3801	nan	nan
2459997	RF_ok	0.00%	0.00%	0.00%	0.00%	-	-	-0.858612	-0.881808	-0.021050	0.117403	-0.536254	-0.939804	0.040102	1.057651	0.5983	0.6163	0.3851	nan	nan
2459996	RF_ok	0.00%	0.00%	0.00%	0.00%	-	-	-0.944304	-0.340702	-0.308127	0.368212	-0.463704	-0.444094	2.435809	2.266051	0.6116	0.6262	0.3952	nan	nan
2459995	RF_ok	0.00%	0.00%	0.00%	0.00%	-	-	-0.957145	-0.858856	-0.210865	0.187460	-0.395064	-0.102744	1.767865	3.039570	0.6035	0.6208	0.3856	nan	nan
2459994	RF_ok	0.00%	0.00%	0.00%	0.00%	-	-	-0.924608	-1.426886	-0.078189	-0.055125	-0.533066	-0.787934	0.548342	2.101214	0.5995	0.6146	0.3798	nan	nan
2459993	RF_ok	100.00%	100.00%	100.00%	0.00%	-	-	249.527816	249.211020	inf	inf	2341.883609	2328.074717	5481.116884	5510.095765	nan	nan	nan	nan	nan
2459991	RF_ok	0.00%	0.00%	0.00%	0.00%	-	-	-0.969043	-1.428249	0.090518	0.173041	-0.638213	-0.747742	1.607234	1.105415	0.5994	0.6061	0.3885	nan	nan
2459990	RF_ok	0.00%	0.00%	0.00%	0.00%	-	-	-0.790033	-1.091970	0.094138	0.221582	-0.677701	-0.891068	2.670533	1.266278	0.5985	0.6089	0.3859	nan	nan
2459989	RF_ok	100.00%	0.00%	0.00%	0.00%	-	-	-0.907021	-1.156749	0.221356	0.144147	-0.597775	-0.888869	5.146638	0.781561	0.5957	0.6118	0.3863	nan	nan
2459988	RF_ok	0.00%	0.00%	0.00%	0.00%	-	-	-0.538486	-0.512409	0.584659	0.734154	-0.614536	-0.689240	0.468608	0.156441	0.5971	0.6120	0.3773	nan	nan
2459987	RF_ok	0.00%	0.00%	0.00%	0.00%	-	-	-1.097753	-1.310431	-0.037075	0.006085	-0.555247	-0.470340	0.353368	1.640042	0.6033	0.6164	0.3790	nan	nan
2459986	RF_ok	0.00%	0.00%	0.00%	0.00%	-	-	-0.851108	-1.232656	-0.014330	0.154558	-0.726053	-0.702570	-0.236128	-0.006545	0.6204	0.6380	0.3472	nan	nan
2459985	RF_ok	0.00%	0.00%	0.00%	0.00%	-	-	-0.587099	-0.449785	0.333312	0.515699	0.018191	-0.894148	-0.327810	-0.472205	0.6059	0.6192	0.3847	nan	nan
2459984	RF_ok	0.00%	0.00%	0.00%	0.00%	-	-	-0.851311	-1.097001	-0.197644	0.233814	-0.705016	-0.532129	0.257540	0.717469	0.6215	0.6377	0.3637	nan	nan
2459983	RF_ok	0.00%	0.00%	0.00%	0.00%	-	-	-0.927614	-1.195696	-0.005744	0.301463	-0.681228	-0.352424	-0.466994	0.084071	0.6262	0.6499	0.3332	nan	nan
2459982	RF_ok	0.00%	0.00%	0.00%	0.00%	-	-	-0.681123	-0.843007	-0.110139	0.014944	-0.964974	-0.720880	-0.703537	-0.845272	0.6792	0.6751	0.2965	nan	nan
2459981	RF_ok	100.00%	0.00%	0.00%	0.00%	-	-	-0.807168	-1.228992	0.240229	0.338551	1.794690	-0.888253	14.918172	1.232610	0.5980	0.6196	0.3818	nan	nan
2459980	RF_ok	0.00%	0.00%	0.00%	0.00%	-	-	-0.940265	-1.011646	0.304722	0.338996	-0.378800	-1.345182	-0.233258	-0.410473	0.6435	0.6522	0.3157	nan	nan
2459979	RF_ok	100.00%	0.00%	0.00%	0.00%	-	-	-1.002603	-1.510145	-0.116260	-0.167763	-0.479852	-0.850688	4.591060	1.290672	0.5953	0.6143	0.3828	nan	nan
2459978	RF_ok	100.00%	0.00%	0.00%	0.00%	-	-	-1.079070	-1.455355	-0.139363	0.027655	-0.546150	-1.141242	4.578635	1.646189	0.5951	0.6134	0.3901	nan	nan
2459977	RF_ok	0.00%	0.00%	0.00%	0.00%	-	-	-0.838157	-1.206380	-0.166803	-0.116770	-0.806167	-1.184203	1.172752	1.613815	0.5595	0.5762	0.3488	nan	nan
2459976	RF_ok	0.00%	0.00%	0.00%	0.00%	-	-	-0.829037	-1.303494	-0.102001	-0.019634	-0.397539	-0.669349	1.076934	1.515408	0.6033	0.6191	0.3796	nan	nan

Antenna Season Report Notebook¶

Antenna 239 Report

Load relevant info from summary CSVs¶

Table 1: Per-Night RTP Summary Info For This Atenna¶

Antenna 239, Node N18:

Load antenna metric spectra and waterfalls from `auto_metrics` notebooks.¶

Figure 1: Antenna autocorrelation metric spectra and waterfalls.¶

Antenna 239: 2460008

Antenna 239: 2460007

Antenna 239: 2459999

Antenna 239: 2459998

Antenna 239: 2459997

Antenna 239: 2459996

Antenna 239: 2459995

Antenna 239: 2459994

Antenna 239: 2459993

Antenna 239: 2459991

Antenna 239: 2459990

Antenna 239: 2459989

Antenna 239: 2459988

Antenna 239: 2459987

Antenna 239: 2459986

Antenna 239: 2459985

Antenna 239: 2459984

Antenna 239: 2459983

Antenna 239: 2459982

Antenna 239: 2459981

Antenna 239: 2459980

Antenna 239: 2459979

Antenna 239: 2459978

Antenna 239: 2459977

Antenna 239: 2459976

Antenna Season Report Notebook¶

Antenna 239 Report

Load relevant info from summary CSVs¶

Table 1: Per-Night RTP Summary Info For This Atenna¶

Antenna 239, Node N18:

Load antenna metric spectra and waterfalls from auto_metrics notebooks.¶

Figure 1: Antenna autocorrelation metric spectra and waterfalls.¶

Antenna 239: 2460008

Antenna 239: 2460007

Antenna 239: 2459999

Antenna 239: 2459998

Antenna 239: 2459997

Antenna 239: 2459996

Antenna 239: 2459995

Antenna 239: 2459994

Antenna 239: 2459993

Antenna 239: 2459991

Antenna 239: 2459990

Antenna 239: 2459989

Antenna 239: 2459988

Antenna 239: 2459987

Antenna 239: 2459986

Antenna 239: 2459985

Antenna 239: 2459984

Antenna 239: 2459983

Antenna 239: 2459982

Antenna 239: 2459981

Antenna 239: 2459980

Antenna 239: 2459979

Antenna 239: 2459978

Antenna 239: 2459977

Antenna 239: 2459976

Load antenna metric spectra and waterfalls from `auto_metrics` notebooks.¶